import tushare as ts
import pandas as pd
import  os
import shutil
def getOnlineBase():
        print("step1 获取基本面数据")
        basic = ts.get_stock_basics()  # 股票列表
        basic.to_csv('d:/data/whitehorse/basic.csv')
        print("step2 获取当前行情数据")
        # 行情和市值数据
        hq = ts.get_today_all()  # 实时行情
        hq.to_csv('d:/data/whitehorse/today.csv')
        print("step3 获取业绩报告数据")
        # 业绩报告数据（2019年1季度）
        re = ts.get_report_data(2019, 1)  # 业绩报告
        re.to_csv('d:/data/whitehorse/report.csv')
        print("step4 获取hs300的股票名单")
        hs300 = ts.get_hs300s()
        hs300.to_csv('d:/data/whitehorse/hs300.csv')
def parseBasic():
    basic=pd.read_csv('d:/data/whitehorse/basic.csv')
    print('step1基本面数据行:',len(basic))
    hq = pd.read_csv('d:/data/whitehorse/today.csv')
    print('step2当前行情数据:', len(hq))
    re = pd.read_csv('d:/data/whitehorse/report.csv')
    print('step3获取业绩报告:', len(re))
    hs300 = pd.read_csv('d:/data/whitehorse/hs300.csv')
    print('step4获取hs300的股票列表:', len(hs300))
    # 把空值设置为0
    #re.fillna(0)
    esp=basic.esp>0.2
    pe=basic.pe>10
    profit=basic.profit>1
    industry=~basic.industry.isin(['饲料', '食品','家居用品','石油','黄金','服饰','航空','石油开采',
                                   '白酒','区域地产','全国地产', '机场','空运','农业综合','旅游服务',
                                   '百货','塑料','钢加工','仓储物流','日用化工','化工原料','铜'
                                   ])
    # 取并集结果
    allcrit = esp & pe & profit & industry

    selected = basic[allcrit]
    print('first',len(selected))
   # selected=selected[selected.code.isin(list)]
    print('second', len(selected))
   # print(len(selected))
    # q1['code'] = selected['code'].astype(str)
    #  finalRes=q1.drop_duplicates(['code'])
    selected.to_csv('d:/data/whitehorse/basic-final.csv')
    dflist = selected[['code']]
    dflist.to_csv('d:/data/whitehorse/basic-codelist.csv',index=None)
def getOnlineCode():
    dfmaxdate = pd.read_csv('d:/data/whitehorse/sh.csv')
    maxdate = dfmaxdate.date.max()
    f = open('d:/data/whitehorse/basic-codelist.csv', "r")
    lines = f.readlines()  # 读取全部内容
    firstFlag=True;
    for line in lines:
        if firstFlag==True:
            firstFlag=False;
            continue;
        code=line.lstrip().replace('\n', '').zfill(6)
        df = ts.get_hist_data(code)
        df.to_csv('d:/data/whitehorse/stock-' + maxdate + '/' + code + '.csv')
        # 保存备份

        print(code)
def combineCodeData():
    df = pd.read_csv('d:/data/whitehorse/sh.csv')
    maxdate=df.date.max()
    path='d:/data/whitehorse/stock-'+maxdate+'/'
    dirs=os.listdir(path)
    fh = open('d:/data/whitehorse/final-code.csv', 'w', encoding='utf-8')
    title='date,open,high,close,low,volume,price_change,p_change,ma5,ma10,ma20,v_ma5,v_ma10,v_ma20\n'
    fh.write( "code," + title)
    for filename in dirs:
        code=filename.replace(".csv","")
        res='';
        #print(code)
        f = open(path+code+'.csv', "r")
        lines = f.readlines()  # 读取全部内容
        i=0;
        for line in lines:

            i=i+1;
            if(i==2):
                fh.write(code+","+line)
                #print(line)
                break
    fh.close()
def parseCodeData():
    path='d:/data/whitehorse/final-code.csv'
    df = pd.read_csv(path, encoding='gbk')
    cond1=df.v_ma5>df.v_ma10*1.2
    cond2 = df.v_ma10> df.v_ma20*1.2
    cond3=df.ma5>df.ma10
    cond4 = df.ma10 > df.ma20
    allcrit =cond1 & cond2 & cond3 & cond4
    selected = df[allcrit]
    selected.to_csv('d:/data/whitehorse/final-code2.csv',index=None)
    print('usual=',len(selected))
    hs300 = pd.read_csv('d:/data/whitehorse/hs300.csv')
    list = hs300['code']
    selected = selected[selected.code.isin(list)]
    selected.to_csv('d:/data/whitehorse/final-code3.csv',index=None)
    print('hs300=',len(selected))
def mergeData():
    path1='d:/data/whitehorse/basic.csv'
    path2='d:/data/whitehorse/today.csv'
    path12='d:/data/whitehorse/basic-codelist.csv'
    path3='d:/data/whitehorse/final-code2.csv'
    path4='d:/data/whitehorse/final-code4.csv'
   # print(path)
    basic = pd.read_csv(path1)
    codelist = pd.read_csv(path12)
    today = pd.read_csv(path2)
    finalcode= pd.read_csv(path3)

    basic = basic.set_index('code')
    codelist= codelist.set_index('code')
    today = today.set_index('code')
    finalcode = finalcode.set_index('code')
    # 合并表格
    basic = basic.merge(today, left_index=True, right_index=True)
    basic = basic.merge(codelist, left_index=True, right_index=True)
    mergedf=finalcode.merge(basic, left_index=True, right_index=True)
    mergedf.to_csv(path4)
    print('final num=',len(mergedf))
def startup(tag='offline'):
#step1 网络请求 startup(1)

    if tag=='online':
        getshData()
        getOnlineBase()
        parseBasic()
        backupCodeData()
        combineCodeData()
        parseCodeData()
        mergeData()
    else:
        #getOnlineBase()
        parseBasic()
        backupCodeData()
        combineCodeData()
        parseCodeData()
        mergeData()
def getshData():
    df = ts.get_hist_data('sh')
    # 保存备份
    df.to_csv('d:/data/whitehorse/sh.csv')

def backupCodeData():
    df = pd.read_csv('d:/data/whitehorse/sh.csv')
    maxdate=df.date.max()
    path = 'd:/data/whitehorse/stock-'+maxdate
    if os.path.exists(path):
        print('文件存在不重新读取')
    else:
        print('no exists')
        os.makedirs(path)
        getOnlineCode()


#startup('offline')
startup('online')
#getshData()
backupCodeData()